import requests
from bs4 import BeautifulSoup
import pandas as pd
import time
import random
import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus'] = False
from collections import Counter
import re

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0 Safari/537.36"
}

url = "https://news.sina.com.cn/"

response = requests.get(url, headers=headers)
response.encoding = 'utf-8'

soup = BeautifulSoup(response.text, 'html.parser')

news_list = []

for a in soup.find_all('a', href=True):
    title = a.get_text().strip()
    link = a['href']

    if title and re.match(r"^https://news\.sina\.com\.cn/.*\d{4}-\d{2}-\d{2}.*\.shtml$", link):
        data_match = re.search(r"(\d{4}-\d{2}-\d{2})", link)
        date = data_match.group(1)
        news_list.append([title, link, date])

    if len(news_list) >= 30:
        break

    time.sleep(random.uniform(0.5, 1.5))

df = pd.DataFrame(news_list, columns=['标题', '链接', '日期'])
df.to_csv('data/sina_news.csv', index=False, encoding='utf-8-sig')

df=pd.read_csv('data/sina_news.csv')
print(f"\n 基本信息")
print(f"-总新闻数：{len(df)}条")
print(f"-日期范围: {df['日期'].min()} 到 {df['日期'].max()}")
print(f"-涉及天数: {df['日期'].nunique()} 天")

df['标题长度'] = df['标题'].apply(len)
print("\n 标题长度分析:")
print(f"-平均标题长度: {df['标题长度'].mean():.1f} 字符")
print(f"-最短标题: {df['标题长度'].min()} 字符 - \"{df.loc[df['标题长度'].idxmin(), '标题']}\"")
print(f"-最长标题: {df['标题长度'].max()} 字符 - \"{df.loc[df['标题长度'].idxmax(), '标题']}\"")

keywords = ['中国', '美国', '经济', '政治', '国际', '社会', '科技']
print("\n 关键词出现统计:")
for keyword in keywords:
    count = sum(keyword in title for title in df['标题'])
    if count > 0:
        print(f"   - {keyword}: {count} 次")

def get_news_category(link):
    """从链接中提取新闻分类"""
    pattern = r'https://news\.sina\.com\.cn/([a-z]+)/'
    match = re.search(pattern, link)
    return match.group(1) if match else '其他'

df['分类'] = df['链接'].apply(get_news_category)

category_mapping = {
    'c': '国内新闻',
    'w': '国际新闻',
    'zx': '最新资讯',
    'gov': '政务新闻',
    'o': '其他新闻',
    'zl': '专栏',
    'photo': '图片新闻',
    'video': '视频新闻',
    'ent': '娱乐',
    'sports': '体育',
    'finance': '财经',
    'tech': '科技',
    'auto': '汽车',
    'house': '房产'
}
df['分类名称'] = df['分类'].map(category_mapping).fillna('其他新闻')
print("\n新闻分类分布:")
category_counts = df['分类名称'].value_counts()
for category, count in category_counts.items():
    print(f"   - {category}: {count} 条")

# 标题长度直方图
plt.figure(figsize=(10, 6))
plt.hist(df['标题长度'], bins=15, color='skyblue', edgecolor='black', alpha=0.7)
plt.title('新闻标题长度分布', fontsize=16, fontweight='bold')
plt.xlabel('标题长度（字符数）')
plt.ylabel('新闻数量')
plt.tight_layout()
plt.savefig('标题长度直方图.png', dpi=300)
plt.show()

# 关键词出现统计柱状图
keyword_counts = [sum(keyword in title for title in df['标题']) for keyword in keywords]
plt.figure(figsize=(10, 6))
bars = plt.bar(keywords, keyword_counts, color='lightcoral', alpha=0.7)
plt.title('关键词在新闻标题中出现次数', fontsize=16, fontweight='bold')
plt.xlabel('关键词')
plt.ylabel('出现次数')
plt.xticks(rotation=45)
for bar, count in zip(bars, keyword_counts):
    plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.1,
             str(count), ha='center', va='bottom')

plt.tight_layout()
plt.savefig('关键词出现统计柱状图.png', dpi=300)
plt.show()


# 分类饼图
plt.figure(figsize=(10, 8))
colors = ['#ff9999', '#66b3ff', '#99ff99', '#ffcc99', '#c2c2f0', '#ffb3e6']
plt.pie(category_counts.values, labels=category_counts.index, autopct='%1.1f%%',
        colors=colors, startangle=90)
plt.title('新闻分类分布', fontsize=16, fontweight='bold')
plt.tight_layout()
plt.savefig('分类饼图.png', dpi=300)
plt.show()
